{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "\n",
    "def bias_variable(shape):\n",
    "    initial = tf.constant(0.0, shape=shape)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "\n",
    "def conv2d(x, W):\n",
    "    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding=\"SAME\")\n",
    "\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding=\"SAME\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cnn():\n",
    "    mnist = input_data.read_data_sets(\"../data/MNIST_data/\", one_hot=True)\n",
    "    x = tf.placeholder(tf.float32, [None, 28 * 28])\n",
    "    y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "    x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "    # convolution 1\n",
    "    W_conv1 = weight_variable([5, 5, 1, 32])\n",
    "    b_conv1 = bias_variable([32])\n",
    "    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
    "\n",
    "    # max pool 1\n",
    "    h_pool1 = max_pool_2x2(h_conv1)\n",
    "\n",
    "    # convolution 2\n",
    "    W_conv2 = weight_variable([5, 5, 32, 64])\n",
    "    b_conv2 = bias_variable([64])\n",
    "    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
    "\n",
    "    # max pool 2\n",
    "    h_pool2 = max_pool_2x2(h_conv2)\n",
    "    h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])\n",
    "\n",
    "    # full connect 1\n",
    "    W_fc1 = weight_variable([7 * 7 * 64, 1024])\n",
    "    b_fc1 = bias_variable([1024])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)\n",
    "\n",
    "    # drop out\n",
    "    keep_prob = tf.placeholder(tf.float32)\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "    # softmax\n",
    "    w_softmax = weight_variable([1024, 10])\n",
    "    b_softmax = bias_variable([10])\n",
    "\n",
    "    y = tf.nn.softmax(tf.matmul(h_fc1_drop, w_softmax) + b_softmax)\n",
    "    cross_entropy = -tf.reduce_sum(y_ * tf.log(y))\n",
    "\n",
    "    update = tf.train.GradientDescentOptimizer(0.0001).minimize(cross_entropy)\n",
    "\n",
    "    correct_prediction = tf.equal(tf.arg_max(y, 1), tf.arg_max(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.initialize_all_variables())\n",
    "        start = time.time()\n",
    "        for i in range(50000):\n",
    "            xs, ys = mnist.train.next_batch(50)\n",
    "            if i % 1000 == 0:\n",
    "                timecos = time.time() - start\n",
    "                start = time.time()\n",
    "                print(\"Step: {:d},Accuracy: {:.3f}, timeCos: {:.1f}\".\n",
    "                      format(i,\n",
    "                             sess.run(accuracy,\n",
    "                                      feed_dict={x: mnist.validation.images,\n",
    "                                                 y_: mnist.validation.labels,\n",
    "                                                 keep_prob: 1.0\n",
    "                                                 }),\n",
    "                             timecos * 1000 / 50\n",
    "                             )\n",
    "                      )\n",
    "            sess.run(update,\n",
    "                     feed_dict={x: xs, y_: ys, keep_prob: 0.5})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ../data/MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting ../data/MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting ../data/MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting ../data/MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From <ipython-input-3-724bb926613d>:43: arg_max (from tensorflow.python.ops.gen_math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `argmax` instead\n",
      "WARNING:tensorflow:From <ipython-input-3-724bb926613d>:43: arg_max (from tensorflow.python.ops.gen_math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `argmax` instead\n",
      "WARNING:tensorflow:From /usr/lib/python3.5/site-packages/tensorflow/python/util/tf_should_use.py:175: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "Step: 0,Accuracy: 0.065, timeCos: 2.1\n",
      "Step: 1000,Accuracy: 0.949, timeCos: 127.6\n",
      "Step: 2000,Accuracy: 0.965, timeCos: 107.8\n",
      "Step: 3000,Accuracy: 0.972, timeCos: 107.4\n",
      "Step: 4000,Accuracy: 0.977, timeCos: 109.0\n",
      "Step: 5000,Accuracy: 0.979, timeCos: 108.9\n",
      "Step: 6000,Accuracy: 0.981, timeCos: 109.4\n",
      "Step: 7000,Accuracy: 0.982, timeCos: 109.3\n",
      "Step: 8000,Accuracy: 0.983, timeCos: 108.7\n",
      "Step: 9000,Accuracy: 0.984, timeCos: 109.2\n",
      "Step: 10000,Accuracy: 0.985, timeCos: 108.9\n",
      "Step: 11000,Accuracy: 0.986, timeCos: 109.0\n",
      "Step: 12000,Accuracy: 0.986, timeCos: 107.2\n",
      "Step: 13000,Accuracy: 0.987, timeCos: 107.7\n",
      "Step: 14000,Accuracy: 0.988, timeCos: 108.4\n",
      "Step: 15000,Accuracy: 0.988, timeCos: 109.3\n",
      "Step: 16000,Accuracy: 0.988, timeCos: 109.4\n"
     ]
    }
   ],
   "source": [
    "cnn()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
